<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.4.2">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/avatar.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/avatar.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/avatar.png">
  <link rel="mask-icon" href="/images/avatar.png" color="#222">

<link rel="stylesheet" href="/css/main.css">


<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/fancyapps/fancybox@3/dist/jquery.fancybox.min.css">

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"notes.maxwi.com","root":"/","scheme":"Mist","version":"7.8.0","exturl":false,"sidebar":{"position":"right","display":"post","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":false,"show_result":false,"style":null},"back2top":{"enable":true,"sidebar":false,"scrollpercent":false},"bookmark":{"enable":true,"color":"#222","save":"auto"},"fancybox":true,"mediumzoom":false,"lazyload":true,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":true,"nav":null},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":false,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},"path":"search.xml"};
  </script>

  <meta name="description" content="系统环境为Ubuntu 16.04.2 x86_64，已经配置好了Hadoop 2.7。配置Hadoop集群环境可以参考这里 约定将spark安装在&#x2F;usr&#x2F;spark，之前安装的Hadoop在路径&#x2F;usr&#x2F;hadoop，这里直接在已经配置好的三台hadoop上配置spark，依然将master节点做为master及worker，其他2个节点做为worker。Spark官方下载地址：http:&#x2F;&#x2F;">
<meta property="og:type" content="article">
<meta property="og:title" content="Spark 2.1.0 On Hadoop 2.7集群配置过程">
<meta property="og:url" content="http://notes.maxwi.com/2017/03/10/spark-on-yarn-configure/index.html">
<meta property="og:site_name" content="blueyi&#39;s notes">
<meta property="og:description" content="系统环境为Ubuntu 16.04.2 x86_64，已经配置好了Hadoop 2.7。配置Hadoop集群环境可以参考这里 约定将spark安装在&#x2F;usr&#x2F;spark，之前安装的Hadoop在路径&#x2F;usr&#x2F;hadoop，这里直接在已经配置好的三台hadoop上配置spark，依然将master节点做为master及worker，其他2个节点做为worker。Spark官方下载地址：http:&#x2F;&#x2F;">
<meta property="og:locale" content="en_US">
<meta property="article:published_time" content="2017-03-10T02:28:32.000Z">
<meta property="article:modified_time" content="2017-03-10T02:28:32.000Z">
<meta property="article:author" content="blueyi">
<meta property="article:tag" content="linux">
<meta property="article:tag" content="Spark">
<meta name="twitter:card" content="summary">

<link rel="canonical" href="http://notes.maxwi.com/2017/03/10/spark-on-yarn-configure/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'en'
  };
</script>

  <title>Spark 2.1.0 On Hadoop 2.7集群配置过程 | blueyi's notes</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

<link rel="alternate" href="/atom.xml" title="blueyi's notes" type="application/atom+xml">
<link rel="alternate" href="/rss2.xml" title="blueyi's notes" type="application/rss+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="Toggle navigation bar">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">blueyi's notes</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
      <p class="site-subtitle" itemprop="description">Follow Excellence,Success will chase you!</p>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
        <i class="fa fa-search fa-fw fa-lg"></i>
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="main-menu menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-home fa-fw"></i>Home</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="fa fa-th fa-fw"></i>Categories</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-archive fa-fw"></i>Archives</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-tags fa-fw"></i>Tags</a>

  </li>
        <li class="menu-item menu-item-about">

    <a href="/about/" rel="section"><i class="fa fa-user fa-fw"></i>About</a>

  </li>
      <li class="menu-item menu-item-search">
        <a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>Search
        </a>
      </li>
  </ul>
</nav>



  <div class="search-pop-overlay">
    <div class="popup search-popup">
        <div class="search-header">
  <span class="search-icon">
    <i class="fa fa-search"></i>
  </span>
  <div class="search-input-container">
    <input autocomplete="off" autocapitalize="off"
           placeholder="Searching..." spellcheck="false"
           type="search" class="search-input">
  </div>
  <span class="popup-btn-close">
    <i class="fa fa-times-circle"></i>
  </span>
</div>
<div id="search-result">
  <div id="no-result">
    <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
  </div>
</div>

    </div>
  </div>

</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>
  <div class="reading-progress-bar"></div>
  <a role="button" class="book-mark-link book-mark-link-fixed"></a>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content post posts-expand">
            

    
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://notes.maxwi.com/2017/03/10/spark-on-yarn-configure/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/default_avatar.jpg">
      <meta itemprop="name" content="blueyi">
      <meta itemprop="description" content="心怀善意，虛怀若谷！">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="blueyi's notes">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          Spark 2.1.0 On Hadoop 2.7集群配置过程
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2017-03-10 10:28:32" itemprop="dateCreated datePublished" datetime="2017-03-10T10:28:32+08:00">2017-03-10</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Spark/" itemprop="url" rel="index"><span itemprop="name">Spark</span></a>
                </span>
            </span>

          <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>5.2k</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>5 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <p>系统环境为Ubuntu 16.04.2 x86_64，已经配置好了Hadoop 2.7。配置Hadoop集群环境可以参考<a href="http://notes.maxwi.com/2017/03/09/hadoop-configure-ubuntu/">这里</a></p>
<p>约定将spark安装在<code>/usr/spark</code>，之前安装的Hadoop在路径<code>/usr/hadoop</code>，这里直接在已经配置好的三台hadoop上配置spark，依然将master节点做为master及worker，其他2个节点做为worker。<br>Spark官方下载地址：<a target="_blank" rel="noopener" href="http://spark.apache.org/downloads.html">http://spark.apache.org/downloads.html</a></p>
<span id="more"></span>
<h2 id="配置"><a href="#配置" class="headerlink" title="配置"></a>配置</h2><p><strong>下载并安装</strong></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">wget http://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.1.0/spark-2.1.0-bin-hadoop2.7.tgz</span><br><span class="line">tar -zxvf spark-2.1.0-bin-hadoop2.7.tgz</span><br><span class="line">mv spark-2.1.0-bin-hadoop2.7 /usr/spark</span><br></pre></td></tr></table></figure>

<p><strong>配置</strong><br>需要修改的文件在路径<code>/usr/spark/conf</code>下，这里我们只需要简单修改<code>spark-env.sh</code>和<code>slaves</code>，更多参数可以参见官方文档。<br><strong>spark-env.sh</strong><br>将官方提供的模板复制一份</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">cp spark-env.sh.template spark-env.sh</span><br></pre></td></tr></table></figure>
<p>添加如下内容：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">export HADOOP_CONF_DIR=/usr/hadoop/etc/hadoop</span><br><span class="line">export SPARK_MASTER_HOST=master</span><br></pre></td></tr></table></figure>
<p><strong>slaves</strong><br>将官方提供的模板复制一份</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">mv slaves.template slaves</span><br></pre></td></tr></table></figure>
<p>添加如下内容：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">localhost</span><br><span class="line">slave1</span><br><span class="line">slave2</span><br></pre></td></tr></table></figure>

<p>如果环境变量配置的都没有问题的话，现在可以运行以下官方示例计算pi值（在路径<code>/usr/spark</code>下执行）：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./bin/run-example SparkPi 10</span><br></pre></td></tr></table></figure>
<p>输出内容的最后面为：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">17/03/11 00:28:31 INFO ui.SparkUI: Stopped Spark web UI at http://192.168.1.124:4040</span><br><span class="line">17/03/11 00:28:31 INFO spark.MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!</span><br><span class="line">17/03/11 00:28:31 INFO memory.MemoryStore: MemoryStore cleared</span><br><span class="line">17/03/11 00:28:31 INFO storage.BlockManager: BlockManager stopped</span><br><span class="line">17/03/11 00:28:31 INFO storage.BlockManagerMaster: BlockManagerMaster stopped</span><br><span class="line">17/03/11 00:28:31 INFO scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!</span><br><span class="line">17/03/11 00:28:31 INFO spark.SparkContext: Successfully stopped SparkContext</span><br><span class="line">17/03/11 00:28:31 INFO util.ShutdownHookManager: Shutdown hook called</span><br><span class="line">17/03/11 00:28:31 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-8e83fd0b-bc88-4452-998a-9ca4300097b8</span><br></pre></td></tr></table></figure>
<p>往上翻就会看到<code>Pi is roughly 3.1413151413151414</code></p>
<p>为了方便执行spark-shell，将spark的bin路径添加到PATH，修改<code>.bashrc</code>添加如下内容：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">Spark</span></span><br><span class="line">export SPARK_HOME=/usr/spark</span><br><span class="line">PATH=$PATH:$SPARK_HOME/bin</span><br><span class="line">export PATH</span><br></pre></td></tr></table></figure>

<p>将spark拷贝到其他节点：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">scp -r /usr/spark slave1:/usr/</span><br><span class="line">scp -r /usr/spark slave1:/usr/</span><br></pre></td></tr></table></figure>
<p>同样将修改的<code>.bashrc</code>拷贝到其他节点，这里spark的环境变量并非必须，只是为了方便执行spark目录下bin中的程序</p>
<h2 id="启动spark集群"><a href="#启动spark集群" class="headerlink" title="启动spark集群"></a>启动spark集群</h2><p>首先启动HDFS和yarn：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">/usr/hadoop/sbin/start-dfs.sh</span><br><span class="line">/usr/hadoop/sbin/start-yarn.sh</span><br></pre></td></tr></table></figure>

<p>为了方便查看完成的历史任务，启动jobhistory：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">/usr/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver</span><br></pre></td></tr></table></figure>

<p>启动spark，为了省事，一次性启动所有master和worker：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">usr/spark/sbin/start-all.sh</span><br></pre></td></tr></table></figure>
<p>现在查看master上的java进程信息：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">1377 NameNode</span><br><span class="line">5235 Worker</span><br><span class="line">6997 Jps</span><br><span class="line">5461 SparkSubmit</span><br><span class="line">5096 Master</span><br><span class="line">3593 ResourceManager</span><br><span class="line">3737 NodeManager</span><br><span class="line">2362 JobHistoryServer</span><br><span class="line">1533 DataNode</span><br><span class="line">1726 SecondaryNameNode</span><br></pre></td></tr></table></figure>
<p>worker上的信息：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">2577 Worker</span><br><span class="line">1267 DataNode</span><br><span class="line">2787 Jps</span><br><span class="line">2159 NodeManager</span><br></pre></td></tr></table></figure>
<p>可以查看web UI：<a target="_blank" rel="noopener" href="http://192.168.1.187:8080">http://192.168.1.187:8080</a><br>如果显示的<code>Alive Worker</code>数量为3以及内容和CPU核数都与实际相符，则表示spark已经配置并启动完成<br>该页面前面的REST URL即是在cluster模式提交spark任务所需要的地址</p>
<h2 id="验证"><a href="#验证" class="headerlink" title="验证"></a>验证</h2><p><strong>YARN方式计算Pi</strong><br>可以在任何节点上运行以下命令，注意<code>--master</code>替换成你自己的</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">/usr/spark/bin/spark-submit \</span><br><span class="line">  --class org.apache.spark.examples.SparkPi \</span><br><span class="line">  --master spark://wyl-node1:6066 \</span><br><span class="line">  --deploy-mode cluster \</span><br><span class="line">  --supervise \</span><br><span class="line">  --executor-memory 1G \</span><br><span class="line">  --total-executor-cores 10 \</span><br><span class="line">  /usr/spark/examples/jars/spark-examples_2.11-2.1.0.jar \</span><br><span class="line">  100000</span><br></pre></td></tr></table></figure>
<p>可以根据自己的配置调整参数，输出中会包含如下信息：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">client token: N/A</span><br><span class="line">diagnostics: N/A</span><br><span class="line">ApplicationMaster host: 192.168.1.187</span><br><span class="line">ApplicationMaster RPC port: 0</span><br><span class="line">queue: default</span><br><span class="line">start time: 1489166056893</span><br><span class="line">final status: UNDEFINED</span><br><span class="line">tracking URL: http://master:8099/proxy/application_1489144101889_0003/</span><br><span class="line">user: root</span><br></pre></td></tr></table></figure>
<p>由于是yarn模式运行，所以无法通过4040端口的页面查看信息，但可以打开提示的相应节点的URL查看到当前的任务执行情况，点击<code>Executors</code>可以看到各work及RDD信息，如果显示的信息与实际相符，说明集群已经完全启动并正在工作<br>本例中YARN的网页地址是<a target="_blank" rel="noopener" href="http://192.168.1.187:8099">http://192.168.1.187:8099</a><br><em>注意Spark Master的WEB UI默认商品是8080，即本例中的网址<a target="_blank" rel="noopener" href="http://192.168.1.187:8080/">http://192.168.1.187:8080/</a></em><br><em>当Spark运行于standalone cluster模式时可以通过4040端口的WEB UI查看详细信息，即本例中的<a target="_blank" rel="noopener" href="http://192.168.1.187:4040">http://192.168.1.187:4040</a>，注意该页面只在有standalone任务运行时才能查看，例如使用spark-shell时</em><br>计算完成之后，输出中会有<code>final status: SUCCEEDED</code><br>通过8099端口的网页，打开相应的任务ID，进入后从Logs里面点击stdout，会看到<code>Pi is roughly 3.141758431417584</code></p>
<p><strong>Standalone Cluster模式运行Spark</strong><br>运行代码为：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">/usr/spark/bin/spark-submit \</span><br><span class="line">  --class org.apache.spark.examples.SparkPi \</span><br><span class="line">  --master spark://wyl-node1:6066 \</span><br><span class="line">  --deploy-mode cluster \</span><br><span class="line">  --supervise \</span><br><span class="line">  --executor-memory 1G \</span><br><span class="line">  --total-executor-cores 10 \</span><br><span class="line">  /usr/spark/examples/jars/spark-examples_2.11-2.1.0.jar \</span><br><span class="line">  10000</span><br></pre></td></tr></table></figure>
<p>通过master的4040端口查看</p>
<p><strong>spark shell</strong><br>这里使用python语言，所以需要用<code>pyspark</code><br>首先将Spark根目录下面的<code>README.md</code>上传到相应用户的HDFS目录下：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hdfs dfs -put README.md /user/root</span><br></pre></td></tr></table></figure>
<p>进入spark shell：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./bin/pyspark</span><br></pre></td></tr></table></figure>
<p>如果刚才设置过spark环境变量，可以直接执行pyspark，输出如下：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line">Python 2.7.12 (default, Nov 19 2016, 06:48:10) </span><br><span class="line">[GCC 5.4.0 20160609] on linux2</span><br><span class="line">Type &quot;help&quot;, &quot;copyright&quot;, &quot;credits&quot; or &quot;license&quot; for more information.</span><br><span class="line">Setting default log level to &quot;WARN&quot;.</span><br><span class="line">To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).</span><br><span class="line">17/03/10 23:50:50 WARN metastore.ObjectStore: Failed to get database global_temp, returning NoSuchObjectException</span><br><span class="line">Welcome to</span><br><span class="line">      ____              __</span><br><span class="line">     / __/__  ___ _____/ /__</span><br><span class="line">    _\ \/ _ \/ _ `/ __/  &#x27;_/</span><br><span class="line">   /__ / .__/\_,_/_/ /_/\_\   version 2.1.0</span><br><span class="line">      /_/</span><br><span class="line"></span><br><span class="line">Using Python version 2.7.12 (default, Nov 19 2016 06:48:10)</span><br><span class="line">SparkSession available as &#x27;spark&#x27;.</span><br></pre></td></tr></table></figure>
<p>输出中的那个警告应该是没有安装HIVE或HBase引起的。<br>下面就是正常的python语言：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">&gt;&gt;&gt; </span>textFile = sc.textFile(<span class="string">&quot;README.md&quot;</span>)</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>textFile.count()  <span class="comment">#统计RDD中的元素数</span></span><br><span class="line"><span class="number">104</span></span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>textFile.first()  <span class="comment">#返回RDD中的第一个元素</span></span><br><span class="line"><span class="string">u&#x27;# Apache Spark&#x27;</span></span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>linesWithSpark = textFile.<span class="built_in">filter</span>(<span class="keyword">lambda</span> line: <span class="string">&quot;Spark&quot;</span> <span class="keyword">in</span> line)</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>textFile.<span class="built_in">filter</span>(<span class="keyword">lambda</span> line: <span class="string">&quot;Spark&quot;</span> <span class="keyword">in</span> line).count() <span class="comment"># 包含`Spark`的行数</span></span><br><span class="line"><span class="number">20</span></span><br></pre></td></tr></table></figure>
<p>如果这里都没有问题，说明Spark已经可以正常工作，剩下的就是学习Spark了</p>
<p><strong>默认配置情况下spark从非master节点启动会有web ui相关的错误提示，直接从master节点关闭spark之后再重新启动即可</strong></p>
<p>参考：<br>1.Spark官网：<a target="_blank" rel="noopener" href="http://spark.apache.org/">http://spark.apache.org/</a><br>2.Running Spark on YARN: <a target="_blank" rel="noopener" href="http://spark.apache.org/docs/latest/running-on-yarn.html">http://spark.apache.org/docs/latest/running-on-yarn.html</a><br>3.Submitting Applications：<a target="_blank" rel="noopener" href="http://spark.apache.org/docs/latest/submitting-applications.html">http://spark.apache.org/docs/latest/submitting-applications.html</a><br>4.Quora-<a target="_blank" rel="noopener" href="https://www.quora.com/How-do-I-set-up-Apache-Spark-with-Yarn-Cluster">How do I set up Apache Spark with Yarn Cluster?</a></p>

    </div>

    
    
    
        

  <div class="followme">
    <p>Welcome to my other publishing channels</p>

    <div class="social-list">

        <div class="social-item">
          <a target="_blank" class="social-link" href="/atom.xml">
            <span class="icon">
              <i class="fa fa-rss"></i>
            </span>

            <span class="label">RSS</span>
          </a>
        </div>
    </div>
  </div>


      <footer class="post-footer">
          <div class="post-tags">
              <a href="/tags/linux/" rel="tag"># linux</a>
              <a href="/tags/Spark/" rel="tag"># Spark</a>
          </div>

        


        
    <div class="post-nav">
      <div class="post-nav-item">
    <a href="/2017/03/09/hadoop-configure-ubuntu/" rel="prev" title="Ubuntu 16.04下配置Hadoop 2.7.3集群">
      <i class="fa fa-chevron-left"></i> Ubuntu 16.04下配置Hadoop 2.7.3集群
    </a></div>
      <div class="post-nav-item">
    <a href="/2017/03/20/python-UnicodeEncodError/" rel="next" title="python 2.7 UnicodeEncodError">
      python 2.7 UnicodeEncodError <i class="fa fa-chevron-right"></i>
    </a></div>
    </div>
      </footer>
    
  </article>
  
  
  



          </div>
          
    <div class="comments" id="gitalk-container"></div>

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          Table of Contents
        </li>
        <li class="sidebar-nav-overview">
          Overview
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#%E9%85%8D%E7%BD%AE"><span class="nav-number">1.</span> <span class="nav-text">配置</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E5%90%AF%E5%8A%A8spark%E9%9B%86%E7%BE%A4"><span class="nav-number">2.</span> <span class="nav-text">启动spark集群</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E9%AA%8C%E8%AF%81"><span class="nav-number">3.</span> <span class="nav-text">验证</span></a></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image" alt="blueyi"
      src="/images/default_avatar.jpg">
  <p class="site-author-name" itemprop="name">blueyi</p>
  <div class="site-description" itemprop="description">心怀善意，虛怀若谷！</div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">104</span>
          <span class="site-state-item-name">posts</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">26</span>
        <span class="site-state-item-name">categories</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">68</span>
        <span class="site-state-item-name">tags</span></a>
      </div>
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/blueyi" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;blueyi" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
      </span>
  </div>


  <div class="links-of-blogroll motion-element">
    <div class="links-of-blogroll-title"><i class="fa fa-link fa-fw"></i>
      Links
    </div>
    <ul class="links-of-blogroll-list">
        <li class="links-of-blogroll-item">
          <a href="http://maxwi.com/" title="http:&#x2F;&#x2F;maxwi.com" rel="noopener" target="_blank">Maxwi</a>
        </li>
    </ul>
  </div>

      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        

<div class="copyright">
  
  &copy; 2014 – 
  <span itemprop="copyrightYear">2022</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">blueyi</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-chart-area"></i>
    </span>
    <span title="Symbols count total">750k</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-coffee"></i>
    </span>
    <span title="Reading time total">11:22</span>
</div>
  <div class="powered-by">Powered by <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://mist.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Mist</a>
  </div>

        








      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/pjax/pjax.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/jquery@3/dist/jquery.min.js"></script>
  <script src="//cdn.jsdelivr.net/gh/fancyapps/fancybox@3/dist/jquery.fancybox.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/lozad@1/dist/lozad.min.js"></script>

<script src="/js/utils.js"></script>


<script src="/js/schemes/muse.js"></script>


<script src="/js/next-boot.js"></script>

<script src="/js/bookmark.js"></script>

  <script>
var pjax = new Pjax({
  selectors: [
    'head title',
    '#page-configurations',
    '.content-wrap',
    '.post-toc-wrap',
    '.languages',
    '#pjax'
  ],
  switches: {
    '.post-toc-wrap': Pjax.switches.innerHTML
  },
  analytics: false,
  cacheBust: false,
  scrollTo : !CONFIG.bookmark.enable
});

window.addEventListener('pjax:success', () => {
  document.querySelectorAll('script[data-pjax], script#page-configurations, #pjax script').forEach(element => {
    var code = element.text || element.textContent || element.innerHTML || '';
    var parent = element.parentNode;
    parent.removeChild(element);
    var script = document.createElement('script');
    if (element.id) {
      script.id = element.id;
    }
    if (element.className) {
      script.className = element.className;
    }
    if (element.type) {
      script.type = element.type;
    }
    if (element.src) {
      script.src = element.src;
      // Force synchronous loading of peripheral JS.
      script.async = false;
    }
    if (element.dataset.pjax !== undefined) {
      script.dataset.pjax = '';
    }
    if (code !== '') {
      script.appendChild(document.createTextNode(code));
    }
    parent.appendChild(script);
  });
  NexT.boot.refresh();
  // Define Motion Sequence & Bootstrap Motion.
  if (CONFIG.motion.enable) {
    NexT.motion.integrator
      .init()
      .add(NexT.motion.middleWares.subMenu)
      .add(NexT.motion.middleWares.postList)
      .bootstrap();
  }
  NexT.utils.updateSidebarPosition();
});
</script>




  
  <script data-pjax>
    (function(){
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x=document.getElementsByTagName("link");
		//Find the last canonical URL
		if(x.length > 0){
			for (i=0;i<x.length;i++){
				if(x[i].rel.toLowerCase() == 'canonical' && x[i].href){
					canonicalURL=x[i].href;
				}
			}
		}
    //Get protocol
	    if (!canonicalURL){
	    	curProtocol = window.location.protocol.split(':')[0];
	    }
	    else{
	    	curProtocol = canonicalURL.split(':')[0];
	    }
      //Get current URL if the canonical URL does not exist
	    if (!canonicalURL) canonicalURL = window.location.href;
	    //Assign script content. Replace current URL with the canonical URL
      !function(){var e=/([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,r=canonicalURL,t=document.referrer;if(!e.test(r)){var n=(String(curProtocol).toLowerCase() === 'https')?"https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif":"//api.share.baidu.com/s.gif";t?(n+="?r="+encodeURIComponent(document.referrer),r&&(n+="&l="+r)):r&&(n+="?l="+r);var i=new Image;i.src=n}}(window);})();
  </script>




  
<script src="/js/local-search.js"></script>













    <div id="pjax">
  

  

<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.css">

<script>
NexT.utils.loadComments(document.querySelector('#gitalk-container'), () => {
  NexT.utils.getScript('//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.js', () => {
    var gitalk = new Gitalk({
      clientID    : '0f8243eb2c8b2207980f',
      clientSecret: 'd159633a33519d3b7a48b0ca729032f7d1f38a41',
      repo        : 'notes',
      owner       : 'blueyi',
      admin       : ['blueyi'],
      id          : '26ca5ad53bb350b947375cea7ccd8c8e',
        language: '',
      distractionFreeMode: true
    });
    gitalk.render('gitalk-container');
  }, window.Gitalk);
});
</script>

    </div>
</body>
</html>
